﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;

namespace PositionalIndexer
{
    public class Tokenizer
    {
        public int TokenID { get; set; }
        public Dictionary<int, Token> TokenDictionary;

        public Tokenizer()
        {
            TokenID = 0;
            TokenDictionary = new Dictionary<int, Token>();
        }

        public void Tokenize(int docID, string str)
        {
            string[] cleanString = Regex.Replace(str, "[^A-Za-z0-9]", " ").Split(' ');

            int position = 0;
            foreach (string s in cleanString)
            {
                if (s != "") // word & number
                {
                    position++;
                }
                if (Regex.Replace(s, "[^A-Za-z]", "") != "") // only word
                {
                    TokenID++;
                    TokenDictionary.Add(TokenID, new Token(Regex.Replace(s.ToLower(), @"[\d-]", string.Empty), docID, position)); //Case folding                 
                }
            }
        }
    }
}